# -*- coding: utf-8 -*-
import pycommonlib as pyclib
import workerpool
import traceback
import datetime, time
import re, os
import hashlib, urllib

from pymongo import Connection
from termcolor  import cprint

LOCAL_PATH = '/home/hoangnamhai/HarvestedData/mobile9'
#MONGO_SERVER    = '27.0.12.106'   
MONGO_SERVER    = 'localhost'   
MONGO_PORT      = 27017
DATABASE        = 'mobile9'
PREFIX      = '/uploads/mobile9' 
SITE_URL    = 'http://gallery.mobile9.com/c/240-320-wallpapers'
PAGE_URL    = 'http://gallery.mobile9.com/c/240-320-wallpapers/1'
BASE_SITE   = 'wallpaper.mobile9.com'
MAX_COUNT   = 100000
CONNECT     = Connection(MONGO_SERVER, MONGO_PORT)
DB          = CONNECT[DATABASE]
#IMAGE_COLLECTION       = DB['images']
IMAGE_COLLECTION       = DB['article']
gbcount = 0
flgCopy             = pyclib.getArgs()
ssh = None; sftp = None
if flgCopy!=None:
    ssh     = pyclib.createSSHClient('mana.vn', 22, 'daudm', '1p%^IRg')
    sftp    = ssh.open_sftp()

conn2       = Connection('beta.mana.vn', 27017)
db2         = conn2['my_database']
def getAuthor(name='crawler'):
    try:
        collection    = db2['backend_user']
        result        = collection.find_one({'username': name}, {})
        if result==None: result = collection.find_one({'username': 'Hindua88'}, {})
        if result!=None:
            return result['_id']
    except:
        traceback.print_exc()

def getRootLft(name='Wallpapers'):
    try:
        root_id = 0; lft = 0
        category    = db2['category']
        result      = category.find_one({'data': name}, {'root_id': 1, 'lft': 1})
        if result!=None:
            root_id = result['root_id']; lft = result['lft']
        return root_id, lft
    except:
        traceback.print_exc()
root_id, lft = getRootLft()
if root_id==None or root_id==0: cprint('Category chưa tồn tại.', 'red'); pyclib.forceQuit()
author       = getAuthor()
if author==None: cprint('Không tìm thấy author', 'red'); pyclib.forceQuit()
SCREEN = [
          '144-176', 
          '160-100', '160-160', '160-240', 
          '172-240',
          '176-192', '176-208', '176-220', '176-240',
          '200-400',
          '208-104', '208-208', '208-320',
          '220-220', '220-176', 
          '228-325',
          '240-240', '240-260', '240-160', '240-320', '240-350', '240-1400', '240-440', '240-480',
          '272-480', 
          '320-240', '320-256', '320-320', '320-400', 
          '360-400', '360-480', '360-640',
          '400-240', 
          '480-320', '480-400', '480-640', '480-800',
          '640-240', '640-320', '640-480', '640-960',
          '800-400', '800-480',
          '960-320', '960-480', '960-800']
CATEGORIES = {   
                 'Vietnam'  :           {'cid' : 'VN',  'category': ''},
                 'Animal':              {'cid' : '1',   'category': ''},
                 'Abstract':            {'cid' : '2',   'category': ''},
                 'Brand':               {'cid' : '3',   'category': ''},
                 'Automotive':          {'cid' : '4',   'category': ''},
                 'Cartoon':             {'cid' : '5',   'category': ''},
                 'Computer':            {'cid' : '6',   'category': ''},
                 'Games':               {'cid' : '7',   'category': ''},
                 'Landscape':           {'cid' : '8',   'category': ''},
                 'Miscellaneous':       {'cid' : '9',   'category': ''},
                 'Movie':               {'cid' : '10',  'category': ''},
                 'Nature':              {'cid' : '11',  'category': ''},
                 'People':              {'cid' : '12',  'category': ''},
                 'Sport':               {'cid' : '13',  'category': ''},
                 'Technology':          {'cid' : '14',  'category': ''},
                 'Original':            {'cid' : '15',  'category': ''},
                 'Sci-Fi & Fantasy':    {'cid' : '24',  'category': ''},
                 'Festive':             {'cid' : '56',  'category': ''},
                 'Funny':               {'cid' : '100', 'category': ''},
                 'Sexy':                {'cid' : '102', 'category': ''},
                 'Signs & Sayings':     {'cid' : '101', 'category': ''},
               }
def getPathWithName(src):
    ''' Hàm lấy tên ảnh ex: abcd
        và đường dẫn ex: a/b/c/d
    '''
    try:
        if type(src).__name__ == 'unicode': src = src.encode('utf-8')
        md5str = hashlib.md5(src).hexdigest()
        path = '{0}/{1}/{2}/{3}'.format(md5str[0], md5str[1], md5str[2], md5str[3])
        return md5str, path
    except:
        traceback.print_exc()
        return None, None

def processWallpaper(link, title, subcat, screen):
    try:
        if link==None or link=='': return
        cprint('Process: ' + link, 'green')
        #hashLink        =   getIdWithLink(link)
        hashLink         =   pyclib.getMd5(link)
        check_exists = IMAGE_COLLECTION.find_one({'hashLink': hashLink})
        if check_exists==None:
            cprint('hashLink: ' + str(hashLink), 'yellow')
            if hashLink==None: return
            tags = []; flgTag = False; title = ''

            tree        =   pyclib.getXMLTree(link)
            if tree==None: cprint('Không build được tree.', 'red'); return
            contentNode	=   tree.xpath('//div[@id="content"]//div[@class="floated_left w660"]')
            if len(contentNode) > 0: primaryNode	= contentNode[0]
            else: cprint('Sai XPath => không thể lấy download được game.', 'red'); return
            titleNode   	= primaryNode.xpath('./h1')
            if len(titleNode)>0: title = pyclib.getStringWithNode(titleNode[0])
            '''
            descNode 	= primaryNode.xpath('./div[@class="floated_left w420"]/div[@id="tabs"]/div[@id="tab-1"]')
            if len(descNode)>0: 
                intro = pyclib.getStringWithNode(descNode[0]); intro = ' '.join(intro.split())    
            '''
            listTagNode     = primaryNode.xpath('.//div[@id="tabs"]/div[@id="tab-2"]/dl/*')
            for tagNode in listTagNode:
                tagContent  = pyclib.getStringWithNode(tagNode)			
                if tagContent==None or len(tagContent)<2: continue
                if flgTag: 
                    flgTrag = False
                    listTag = tagNode.xpath('.//a')
                    for t in listTag: tags.append(pyclib.getStringWithNode(t));
                    break	
                if tagContent=='Tags': flgTag = True
                
            imgNode     = primaryNode.xpath('./div[@class="floated_left w420"]//div[@class="gallery"]//a')
            if len(imgNode)>0:
                src = imgNode[0].get('href')
                if src==None and src=='': return
                linkImage = src
                result = None; source = file_name = ''; size = 0
                if flgCopy!=None:
                    result, source, file_name, size = pyclib.saveImageWithSCP(linkImage, PREFIX, LOCAL_PATH, ssh, sftp)
                else:
                    result, source, file_name, size     =  pyclib.saveImage(src, PREFIX, LOCAL_PATH)
                if result!=None: 
                    images = {'original_name': file_name, 'size': size, 'source': source}
                    dtnow  = datetime.datetime.now()
                    doc = { 'hashLink': hashLink,
                            'link': link,
                            'content': {'image': images},
                            'resolution': screen, 'date': datetime.datetime.utcnow(), 
                            'title': title, 
                            'source': 'mobile9.com', 
                            'is_active': True, 
                            'tags': tags, 
                            'is_active': True,
                            'subcategory': subcat,
                            'root': root_id,
                            'category': lft,
                            'author_id': author,
                            'type': 'wallpaper',
                            'update': dtnow,
                            'lastupdate': dtnow
                            }
                    IMAGE_COLLECTION.save(doc); print doc; return 0
        else:
            cprint('Wallpaper đã tồn tại trong cơ sỡ dữ liệu', 'red'); return 1
    except:
        traceback.print_exc()

def processPage(lurl, subcat, screen):
    try:
        gbcount = 0
        print 'GLOBAL COUNT', gbcount
        if gbcount>MAX_COUNT: 
            cprint('Dừng chương trình vì quá số trùng lặp cho phép')
            print 'Finished.', datetime.datetime.now();
            pid = os.getpid();  os._exit(1); os.kill(pid, 9)
        cprint('Process page : ' + lurl, 'yellow')
        tree    = pyclib.getXMLTree(lurl)
        listGame   = tree.xpath('//div[@id="content"]//div[@class="gallery"]/a')
        for game in listGame:
            print 'GLOBAL COUNT', gbcount
            if gbcount>MAX_COUNT: 
                cprint('Dừng chương trình vì quá số trùng lặp cho phép')
                print 'Finished.', datetime.datetime.now()
                pid = os.getpid();  os._exit(1); os.kill(pid, 9)
            title   = pyclib.getStringWithNode(game) 
            count   = processWallpaper(game.get('href'), title, subcat, screen)
            if count!=None: gbcount += count 
        return gbcount
    except:
        traceback.print_exc()


def getIdWithLink(link):
    try:
        if link==None or link=='': return 
        m       = pyclib.regexString('/(\d+)/', link)
        if m: return m.group(1)
    except:
        traceback.print_exc()

def getMaxPage(url):
    try:
        maxPage = 1
        tree = pyclib.getXMLTree(url)
        pageNode = tree.xpath('//div[@id="content"]//div[@class="page"]//strong')
        if len(pageNode)>0:
            text = pyclib.getStringWithNode(pageNode[0])
            preg = re.compile(r'(\d+)$')
            m    = preg.search(text)
            if m: maxPage = int(float(m.group(1))) + 1
        return maxPage
    except:
        traceback.print_exc()

def processAll(cat):
    try:
        for item in SCREEN:
            site_url = 'http://gallery.mobile9.com/c/{0}-wallpapers'.format(item)
            screen = item.replace('-', 'x') 
            for i in [1, 3, 2]:
                cCount = 0
                cId  = CATEGORIES[cat]['cid']
                lurl = '{0}{1}{2}{3}'.format(site_url, '/1/?st=' + str(i) + '&fi=', cId, '&ff=on')
                for page in range(1, getMaxPage(lurl)):
                    url = '{0}{1}{2}{3}'.format(site_url, '/' + str(page) + '/?st=' + str(i) + '&fi=', cId, '&ff=on')
                    cnt  =  processPage(url, cat, screen)
                    if cnt!=None: cCount += cnt 
                    if cCount>MAX_COUNT: 
                        cprint('Dừng chương trình vì quá số trùng lặp cho phép')
                        print 'Finished.', datetime.datetime.now();
                        pid = os.getpid();  os._exit(1); os.kill(pid, 9)
    except:
        traceback.print_exc()

if __name__ == '__main__':
    try:
        cprint('start crawler wallpapers.mobile9.com', 'yellow')
        pool = workerpool.WorkerPool(size=8)
        pool.map(processAll, CATEGORIES.keys())
        pool.shutdown(); pool.wait()
        if flgCopy!=None: ssh.close()
        conn2.end_request()
        print 'Finished.', datetime.datetime.now();
        pid = os.getpid();  os._exit(1); os.kill(pid, 9)
    except:
        traceback.print_exc()


